<!DOCTYPE html>
<html lang="en-us">
<head><head>
    <meta name="google-site-verification" content="9vIieCe-Qpd78QOmBl63rGtIVbhY6sYyuxX3j8XWBA4" />
    <meta name="baidu-site-verification" content="LRrmH41lz7" />
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="google-site-verification" content="xBT4GhYoi5qRD5tr338pgPM5OWHHIDR6mNg1a3euekI" />
    <meta name="viewport" content="width=device-width, initial-scale=1">
    
    <meta name="baidu-site-verification" content="HGLXRsUXC4" />

    
    <meta name="baidu-site-verification" content="code-ANZvlnN0Xr" />

    
    <meta name="description" content="由于logstash内存占用较大,灵活性相对没那么好,ELK正在被EFK逐步替代.">
    
    <meta name="keyword"  content="liangyuanpeng|LanLiang|OpenYurt|Knative|Pulsar|Prometheus|Halo||边缘计算kubernetes|Docker|CloudNative|Golang|Rust|Istio|微服务">
    <link rel="shortcut icon" href="img/logo.png">


    
    <meta property="og:image" content="https://res.cloudinary.com/lyp/image/upload/v1581932131/hugo/blog.github.io/you-got-this-lighted-signage-2740954.jpg" />
    <meta name="twitter:image" content="https://res.cloudinary.com/lyp/image/upload/v1581932131/hugo/blog.github.io/you-got-this-lighted-signage-2740954.jpg" />


    <title>Elasticsearch&#43;Fluentd&#43;Kafka搭建分布式日志系统-liangyuanpeng的博客 | liangyuanpeng&#39;s Blog</title>

    <link rel="canonical" href="/post/elasticsearch-fluentd-kafka/">

    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/iDisqus.min.css"/>

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369744/hugo/css/bootstrap.min.css">

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/hux-blog.min.css">

    
    <link rel="stylesheet" href="https://res.cloudinary.com/lyp/raw/upload/v1537369740/hugo/css/syntax.css">

    
    <link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">
    
    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369966/hugo/js/jquery.min.js"></script>

    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369966/hugo/js/bootstrap.min.js"></script>

    
    <script src="https://res.cloudinary.com/lyp/raw/upload/v1537369964/hugo/js/hux-blog.min.js"></script>

    
    <script defer src='https://static.cloudflareinsights.com/beacon.min.js' data-cf-beacon='{"token": "f73bc0092aeb491d89984c0eb5a87ac2"}'></script>
</head>

</head>

<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
    <div class="container-fluid">
        
        <div class="navbar-header page-scroll">
            <button type="button" class="navbar-toggle">
                <span class="sr-only">Toggle navigation</span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
            </button>
            <a class="navbar-brand" href="/">Hi,I`m lan</a>
        </div>

        
        
        <div id="huxblog_navbar">
            <div class="navbar-collapse">
                <ul class="nav navbar-nav navbar-right">
                    <li>
                        <a href="/">Home</a>
                    </li>
                    
                    <li>
                        <a href="categories/cloudnative">cloudnative</a>
                    </li>
                    
                    <li>
                        <a href="categories/devops">devops</a>
                    </li>
                    
                    <li>
                        <a href="categories/iot">iot</a>
                    </li>
                    
                    <li>
                        <a href="categories/kubernetes">kubernetes</a>
                    </li>
                    
                    <li>
                        <a href="categories/tech">tech</a>
                    </li>
                    

                    
		            <li>
                        <a href="search">SEARCH <img src="img/search.png" height="15" style="cursor: pointer;"></a>
		            </li>
                    
                </ul>
            </div>
        </div>
        
    </div>
    
</nav>
<script>
    
    
    
    var $body   = document.body;
    var $toggle = document.querySelector('.navbar-toggle');
    var $navbar = document.querySelector('#huxblog_navbar');
    var $collapse = document.querySelector('.navbar-collapse');

    $toggle.addEventListener('click', handleMagic)
    function handleMagic(e){
        if ($navbar.className.indexOf('in') > 0) {
        
            $navbar.className = " ";
            
            setTimeout(function(){
                
                if($navbar.className.indexOf('in') < 0) {
                    $collapse.style.height = "0px"
                }
            },400)
        }else{
        
            $collapse.style.height = "auto"
            $navbar.className += " in";
        }
    }
</script>




<style type="text/css">
    header.intro-header{
        background-image: url('https://res.cloudinary.com/lyp/image/upload/v1581932131/hugo/blog.github.io/you-got-this-lighted-signage-2740954.jpg')
    }
</style>
<header class="intro-header" >
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <div class="post-heading">
                    <div class="tags">
                       
                       <a class="tag" href="/tags/docker" title="docker">
                           docker
                        </a>
                        
                       <a class="tag" href="/tags/cadvisor" title="cadvisor">
                           cadvisor
                        </a>
                        
                       <a class="tag" href="/tags/kafka" title="kafka">
                           kafka
                        </a>
                        
                       <a class="tag" href="/tags/fluentd" title="fluentd">
                           fluentd
                        </a>
                        
                       <a class="tag" href="/tags/elasticsearch" title="elasticsearch">
                           elasticsearch
                        </a>
                        
                    </div>
                    <h1>Elasticsearch&#43;Fluentd&#43;Kafka搭建分布式日志系统</h1>
                    <h2 class="subheading"></h2>
                    <span  class="meta">Posted by 梁远鹏 on 2020-02-17
                        
                        <span id="busuanzi_container_page_pv">|<span id="busuanzi_value_page_pv"></span><span>
                            <span id="/post/elasticsearch-fluentd-kafka/" class="leancloud_visitors meta_data_item" data-flag-title="">
    <span class="post-meta-item-icon">
      <span class="octicon octicon-eye"></span> 
    </span>
    <i class="fa fa-eye"></i>
    <span class="old-visitors-count" style="display: none;"></span>
    <span class="leancloud-visitors-count"></span>
</span>






                            阅读 </span></span>|<span class="post-date">共1304字</span>，阅读约<span class="more-meta"> 3 分钟</span>
                        
                    </span>
                </div>
            </div>
        </div>
    </div>
</header>




<article>
    <div class="container">
        <div class="row">

            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                post-container">

		
                <header>
                <h2>TOC</h2>
                </header>
                <nav id="TableOfContents">
  <ul>
    <li><a href="#数据采集流程">数据采集流程</a></li>
    <li><a href="#配置文件">配置文件</a>
      <ul>
        <li><a href="#开始部署">开始部署</a></li>
      </ul>
    </li>
  </ul>
</nav>
		
		<h1 id="前言">前言</h1>
<p>由于logstash内存占用较大,灵活性相对没那么好,ELK正在被EFK逐步替代.其中本文所讲的EFK是Elasticsearch+Fluentd+Kafka,实际上K应该是Kibana用于日志的展示,这一块不做演示,本文只讲述数据的采集流程.</p>
<h1 id="前提">前提</h1>
<ol>
<li><a href="https://www.docker.com/get-started">docker</a></li>
<li><a href="https://github.com/docker/compose">docker-compose</a></li>
<li><a href="https://liangyuanpeng.com/post/docker-deploy-kafka/">apache kafka服务</a></li>
</ol>
<h1 id="架构">架构</h1>
<h2 id="数据采集流程">数据采集流程</h2>
<p>数据的产生使用cadvisor采集容器的监控数据并将数据传输到Kafka.</p>
<p>数据的传输链路是这样: Cadvisor-&gt;Kafka-&gt;Fluentd-&gt;elasticsearch</p>
<p><img src="https://res.cloudinary.com/lyp/image/upload/v1581931896/hugo/blog.github.io/fluentd/cadvisor-kafka-fluentd-es.jpg" alt="https://res.cloudinary.com/lyp/image/upload/v1581931896/hugo/blog.github.io/fluentd/cadvisor-kafka-fluentd-es.jpg"></p>
<p>每一个服务都可以横向扩展,添加服务到日志系统中.</p>
<h2 id="配置文件">配置文件</h2>
<p>docker-compose.yml</p>
<pre><code>version: &quot;3.7&quot;

services:
  
  elasticsearch:
   image: elasticsearch:7.5.1
   environment:
    - discovery.type=single-node  #使用单机模式启动
   ports:
    - 9200:9200

  cadvisor:
    image: google/cadvisor
    command: -storage_driver=kafka -storage_driver_kafka_broker_list=192.168.1.60:9092(kafka服务IP:PORT) -storage_driver_kafka_topic=kafeidou
    depends_on:
      - elasticsearch

  fluentd:
   image: lypgcs/fluentd-es-kafka:v1.3.2
   volumes:
    - ./:/etc/fluent
    - /var/log/fluentd:/var/log/fluentd
</code></pre><p>其中:</p>
<ol>
<li>cadvisor产生的数据会传输到192.168.1.60这台机器的kafka服务,topic为kafeidou</li>
<li>elasticsearch指定为单机模式启动(<code>discovery.type=single-node</code>环境变量),单机模式启动是为了方便实验整体效果</li>
</ol>
<p>fluent.conf</p>
<pre><code>#&lt;source&gt;
#  type http
#  port 8888
#&lt;/source&gt;

&lt;source&gt;
  @type kafka
  brokers 192.168.1.60:9092
  format json
  &lt;topic&gt;
    topic     kafeidou
  &lt;/topic&gt;
&lt;/source&gt;

&lt;match **&gt;
  @type copy

#  &lt;store&gt;
#   @type stdout
#  &lt;/store&gt;

  &lt;store&gt;
  @type elasticsearch
  host 192.168.1.60
  port 9200
  logstash_format true
  #target_index_key machine_name
  logstash_prefix kafeidou
  logstash_dateformat %Y.%m.%d   
  
  flush_interval 10s
  &lt;/store&gt;
&lt;/match&gt;

</code></pre><p>其中:</p>
<ol>
<li>
<p>type为copy的插件是为了能够将fluentd接收到的数据复制一份,是为了方便调试,将数据打印在控制台或者存储到文件中,这个配置文件默认关闭了,只提供必要的es输出插件.<br>
需要时可以将<code>@type stdout</code>这一块打开,调试是否接收到数据.</p>
</li>
<li>
<p>输入源也配置了一个http的输入配置,默认关闭,也是用于调试,往fluentd放入数据.<br>
可以在linux上执行下面这条命令:</p>
</li>
</ol>
<pre><code>curl -i -X POST -d 'json={&quot;action&quot;:&quot;write&quot;,&quot;user&quot;:&quot;kafeidou&quot;}' http://localhost:8888/mytag
</code></pre><ol start="3">
<li>target_index_key参数,这个参数是将数据中的某个字段对应的值作为es的索引,例如这个配置文件用的是machine_name这个字段内的值作为es的索引.</li>
</ol>
<h3 id="开始部署">开始部署</h3>
<p>在包含docker-compose.yml文件和fluent.conf文件的目录下执行:<br>
<code>docker-compose up -d</code></p>
<p>在查看所有容器都正常工作之后可以查看一下elasticsearch是否生成了预期中的数据作为验证,这里使用查看es的索引是否有生成以及数据数量来验证:</p>
<pre><code>-bash: -: 未找到命令
[root@master kafka]# curl http://192.168.1.60:9200/_cat/indices?v
health status index                                uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   55a4a25feff6                         Fz_5v3suRSasX_Olsp-4tA   1   1       1            0      4kb            4kb
</code></pre><p>也可以直接在浏览器输入<code> http://192.168.1.60:9200/_cat/indices?v</code>查看结果,会更方便.</p>
<p>可以看到我这里是用了machine_name这个字段作为索引值,查询的结果是生成了一个叫<code>55a4a25feff6</code>的索引数据,生成了1条数据(<code>docs.count</code>)</p>
<p>到目前为止<code>kafka-&gt;fluentd-&gt;es</code>这样一个日志收集流程就搭建完成了.</p>
<p>当然了,架构不是固定的.也可以使用<code>fluentd-&gt;kafka-&gt;es</code>这样的方式进行收集数据.这里不做演示了,无非是修改一下fluentd.conf配置文件,将es和kafka相关的配置做一下对应的位置调换就可以了.</p>
<p>鼓励多看官方文档,在github或fluentd官网上都可以查找到fluentd-es插件和fluentd-kafka插件.</p>


        <h2>微信公众号</h2>
<p>扫描下面的二维码关注我们的微信公众号,第一时间查看最新内容。同时也可以关注我的Github，看看我都在了解什么技术，在页面底部可以找到我的Github。</p>
<img src="https://res.cloudinary.com/lyp/image/upload/v1581729955/hugo/blog.github.io/qrcode_for_sikekafeidou.jpg" alt="wechat-account-qrcode">


                <hr>

                <ul class="pager">
                    
                    <li class="previous">
                        <a href="/post/logstash-logdriver-docker/" data-toggle="tooltip" data-placement="top" title="使用logstash作为docker日志驱动收集日志">&larr; 使用logstash作为docker日志驱动收集日志</a>
                    </li>
                    
                    
                    <li class="next">
                        <a href="/post/jib-springboot-docker-maven/" data-toggle="tooltip" data-placement="top" title="不用安装docker也能构建docker镜像">不用安装docker也能构建docker镜像 &rarr;</a>
                    </li>
                    
                </ul>

            
<script src="https://utteranc.es/client.js"
repo="liangyuanpeng/liangyuanpeng.github.io"
issue-term="title"
theme="github-light"
crossorigin="anonymous"
async>
</script>



            </div>

            


            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                sidebar-container">

                
                <section>
                    <hr class="hidden-sm hidden-xs">
                    <h5><a href="/tags/">FEATURED TAGS</a></h5>
                    <div class="tags">
                     
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/blog" title="blog">
                                blog
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/chirpstack" title="chirpstack">
                                chirpstack
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/ci/cd" title="ci/cd">
                                ci/cd
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/cloudnative" title="cloudnative">
                                cloudnative
                            </a>
                        
                    
                        
                            <a href="/tags/cncf" title="cncf">
                                cncf
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/docker" title="docker">
                                docker
                            </a>
                        
                    
                        
                            <a href="/tags/docker-compose" title="docker-compose">
                                docker-compose
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/fluentd" title="fluentd">
                                fluentd
                            </a>
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/halo" title="halo">
                                halo
                            </a>
                        
                    
                        
                            <a href="/tags/hugo" title="hugo">
                                hugo
                            </a>
                        
                    
                        
                            <a href="/tags/image" title="image">
                                image
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/iot" title="iot">
                                iot
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/kubernetes" title="kubernetes">
                                kubernetes
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/lorawan" title="lorawan">
                                lorawan
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/middleware" title="middleware">
                                middleware
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/mq" title="mq">
                                mq
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/ops" title="ops">
                                ops
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/prometheus" title="prometheus">
                                prometheus
                            </a>
                        
                    
                        
                    
                        
                    
                        
                            <a href="/tags/rpc" title="rpc">
                                rpc
                            </a>
                        
                    
                        
                    
                        
                            <a href="/tags/sofa" title="sofa">
                                sofa
                            </a>
                        
                    
                        
                            <a href="/tags/sofastack" title="sofastack">
                                sofastack
                            </a>
                        
                    
                        
                            <a href="/tags/springboot" title="springboot">
                                springboot
                            </a>
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                        
                    
                    </div>
                </section>



               

                
                
            </div>

            
            
            <div class="
            col-lg- col-lg-offset-2
            col-md-10 col-md-offset-1
            ">
            <section>
                <hr class="hidden-sm hidden-xs">
                

<h2>相关文章</h2>
<ul style="margin-bottom: 25px;">
    
    <li><a href="/post/fluentd-logdrive-docker/">使用fluentd作为docker日志驱动收集日志</a></li>
    
    <li><a href="/post/fluentd-plugin-kafka/">制作带有kafka插件和es插件的fluentd镜像</a></li>
    
    <li><a href="/post/fluentd-quickstart/">fluentd语法入门</a></li>
    
    <li><a href="/post/logstash-logdriver-docker/">使用logstash作为docker日志驱动收集日志</a></li>
    
    <li><a href="/post/ballerina-hello-world/">云原生编程语言ballerina:hello-world</a></li>
    
</ul>

            </section>
            </div>
            


        </div>
    </div>
</article>






<footer>
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <ul class="list-inline text-center">
                   
                    
                    <li>
                        <a target="_blank" href="https://github.com/liangyuanpeng">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
                   
                    
                    <li>
                        <a href="mailto:liangyuanpengem@163.com">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    

                    
                    <li>
                        <a href="https://twitter.com/lan31793328">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-twitter fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li>
                        <a target="_blank" href="https://www.zhihu.com/people/liangyuanpeng">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa  fa-stack-1x fa-inverse">知</i>
                            </span>
                        </a>
                    </li>
		    
                    
                    <li>
                        <a target="_blank" href="https://weibo.com/u/1908782280">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-weibo fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    

                    

		    
                    
                    
                </ul>
		<p class="copyright text-muted">
                    Copyright &copy; Hi,I`m lan , 2020
                    <br>
                    <a href="https://themes.gohugo.io/hugo-theme-cleanwhite">CleanWhite Hugo Theme</a> by <a href="https://zhaohuabing.com">Huabing</a> |
                    <iframe
                        style="margin-left: 2px; margin-bottom:-5px;"
                        frameborder="0" scrolling="0" width="100px" height="20px"
                        src="https://ghbtns.com/github-btn.html?user=zhaohuabing&repo=hugo-theme-cleanwhite&type=star&count=true" >
                    </iframe>
                </p>
            </div>
        </div>
    </div>
</footer>




<script>
    function async(u, c) {
      var d = document, t = 'script',
          o = d.createElement(t),
          s = d.getElementsByTagName(t)[0];
      o.src = u;
      if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
      s.parentNode.insertBefore(o, s);
    }
</script>






<script>
    
    if($('#tag_cloud').length !== 0){
        async("/js/jquery.tagcloud.js",function(){
            $.fn.tagcloud.defaults = {
                
                color: {start: '#bbbbee', end: '#0085a1'},
            };
            $('#tag_cloud a').tagcloud();
        })
    }
</script>


<script>
    async("https://cdnjs.cloudflare.com/ajax/libs/fastclick/1.0.6/fastclick.js", function(){
        var $nav = document.querySelector("nav");
        if($nav) FastClick.attach($nav);
    })
</script>







<script>
    
    var _baId = 'fad9c137f8ce239f9b323d36c871f8e6';

    
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "//hm.baidu.com/hm.js?" + _baId;
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
</script>







</body>
</html>
